import re
# import sys
import logging
import urlparse
import time
import robotparser
import threading
import multiprocessing
from pymongo import MongoClient
from mongotool import MongoQueue
from download import Downloader
from utilities import make_random_useragent

SLEEP_TIME = 3
DEFAULT_AGENT = make_random_useragent()
DEFAULT_HEADERS = {
    "User-Agent": DEFAULT_AGENT,
    "Accept-Encoding": "gzip"
}

# logging.basicConfig(stream=sys.stderr, level=logging.INFO)


def normalize(seed_url, link):
    """Normalize this URL by removing hash and adding domain
    """
    link, _ = urlparse.urldefrag(link)  # remove hash to avoid duplicates
    return urlparse.urljoin(seed_url, link)


def same_domain(url1, url2):
    """Return True if both URL's belong to same domain
    """
    return urlparse.urlparse(url1).netloc == urlparse.urlparse(url2).netloc


def get_robots(url):
    """Initialize robots parser for this domain
    """
    rp = robotparser.RobotFileParser()
    rp.set_url(urlparse.urljoin(url, '/robots.txt'))
    rp.read()
    return rp


def get_links(html):
    """Return a list of links from html
    """
    # a regular expression to extract all links from the webpage
    webpage_regex = re.compile('<a[^>]+href=["\'](.*?)["\']', re.IGNORECASE)
    # link_regex = re.compile('[a-zA-z]+://[^\s]*', re.IGNORECASE)
    link_regex = re.compile(  # filter url with query words and params
        '^(?:([A-Za-z]+):)?(/{0,3})([0-9.\-A-Za-z]+)(?:/([^?#]*))?$', re.IGNORECASE)
    # list of all links from the webpage
    links = []
    links.extend(link for link in webpage_regex.findall(
        html) if re.match(link_regex, link))
    return links


def threaded_crawler(seed_url, delay=5, cache=None, scrape_callback=None, download_callback=None, headers=DEFAULT_HEADERS, proxies=None, num_retries=1, max_threads=10, timeout=60, email_notify=False, remove_proxy=True):
    """Crawl using multiple threads
    """
    client = MongoClient(
        'localhost', 27017, maxPoolSize=50, waitQueueMultiple=10, connect=False,)
    # the queue of URL's that still need to be crawled
    crawl_queue = MongoQueue(client)
    # crawl_queue.clear()
    crawl_queue.push(seed_url)
    D = Downloader(cache=cache, delay=delay, headers=headers,
                   proxies=proxies, num_retries=num_retries, timeout=timeout, email_notify=email_notify, remove_proxy=remove_proxy)

    def process_queue():
        while True:
            # keep track that are processing url
            try:
                url = crawl_queue.pop()
            except KeyError:
                # currently no urls to process
                break
            else:
                html = D(url)
                if download_callback:
                    try:
                        download_callback(url, html, client)
                    except Exception as e:
                        logging.error(
                            '[download_callback] for: {} >>Error: {}'.format(url, e))
                if scrape_callback:
                    try:
                        links = scrape_callback(url, html) or []
                    except Exception as e:
                        logging.error(
                            '[scrape_callback] for: {} >>Error: {}'.format(url, e))
                    else:
                        for link in links:
                            # add this new link to queue
                            crawl_queue.push(normalize(seed_url, link))
                crawl_queue.complete(url)

    # wait for all download threads to finish
    threads = []
    while threads or crawl_queue:
        for thread in threads:
            if not thread.is_alive():
                threads.remove(thread)
        while len(threads) < max_threads and crawl_queue.peek():
            # can start some more threads
            thread = threading.Thread(target=process_queue)
            # set daemon so main thread can exit when receives ctrl-c
            thread.setDaemon(True)
            thread.start()
            threads.append(thread)
        time.sleep(SLEEP_TIME)


def process_crawler(args, num_process=None, **kwargs):
    if not num_process:
        num_process = multiprocessing.cpu_count()
    # pool = multiprocessing.Pool(processes=num_cpus)
    logging.info('Starting {} processes'.format(num_process))
    processes = []
    for i in range(num_process):
        p = multiprocessing.Process(
            target=threaded_crawler, args=[args], kwargs=kwargs)
        # parsed = pool.apply_async(threaded_link_crawler, args, kwargs)
        p.start()
        processes.append(p)
    # wait for processes to complete
    for p in processes:
        p.join()


if __name__ == '__main__':
    pass
